#!/usr/bin/env bash

KUBECTL_URL="https://storage.googleapis.com/kubernetes-release/release"

# copy .env
if ! [ -f .env ];then cp .env.example .env; fi
if ! [ -f coreos/.env ];then cp coreos/.env.example coreos/.env; fi
if ! [ -f systemd/.env ];then cp systemd/.env.example systemd/.env; fi
if ! [ -f nfs-server/.env ];then cp nfs-server/.env.example nfs-server/.env; fi
if ! [ -f nfs-server/exports.txt ];then cp nfs-server/exports.example.txt nfs-server/exports.txt; fi
if ! [ -f wsl2/.env ];then cp wsl2/.env.example wsl2/.env; fi

source ./systemd/.env.example
source ./systemd/.env
source ./.env.example
source ./.env

TMP_ROOT=$HOME/.khs1994-docker-lnmp/caches

################################################################################

if ! [ -f lnmp-k8s ];then if [ -n "$LNMP_PATH" ];then cd $LNMP_PATH/kubernetes > /dev/null; else exit; fi; fi

if [ -n "${DEBUG}" -o -n "${debug}" ];then set -x; fi

EXEC_CMD_DIR=$PWD

# CI/CD
if [ -n "${CI_KUBERNETES_VERSION}" ];then KUBERNETES_VERSION=${CI_KUBERNETES_VERSION};fi

set -e

OS=`uname -s`

std="stderr"
if [ -n "$QUITE" -o -n "$quite" ];then std="null";fi

print_info(){
  echo "" > /dev/${std}
  echo "==> $1" > /dev/${std}
  echo "" > /dev/${std}
}

print_help_info(){
  echo "
Kubernetes version is ${KUBERNETES_VERSION}


Usage: lnmp-k8s COMMAND

Commands:
  kubernetes-server      Download kubernetes server files {--url} {GOOS} {GOARCH} {TYPE:-server} {KUBERNETES_VERSION:-${KUBERNETES_VERSION}}
  pull-k8s-image         Use By developer Only {GOOS} {GOARCH} {TYPE:-server} {KUBERNETES_VERSION:-${KUBERNETES_VERSION}}

  kubectl-install        Install kubectl
  kubectl-info           Get kubectl latest stable version info

  local-install          Install local Kubernetes (Manager by systemd) { CRI: --containerd (default) | --crio | --docker }
  local-up               Print how to start local Kubernetes (Manager by systemd)
  local-cleanup          Cleanup local Kubernetes.

  join                   New worker [NODE_IP] { CRI: --containerd (default) | --crio | --docker } {--hostname=HOST_NAME} {--skip-cp-k8s-bin}

  nfs                    Up NFS server { down | logs }

ENV:
  UPLOAD_TO_CODING=false
  DOWNLOAD_FROM_CODING=true
  CODING_USERNAME
  CODING_TOKEN
"
}

_cp(){
  if ! [ -f $1 ];then echo "$1 not found"; touch $2; return ;fi
  if ! [ -f $2 ];then cp $1 $2 ;fi
}

if [ "$#" = 0 ];then print_help_info;exit;fi

_curl() {
  if [ -n "${DEBUG}" -o -n "${debug}" ];then echo "curl -fL -A curl-lnmp-k8s"; return; fi

  echo "curl -fsSL -A curl-lnmp-k8s "
}

_sudo(){
  command -v sudo > /dev/null && echo sudo || true
}

uname -a | grep -q microsoft && IS_WSL2=1 || true

$(_sudo) systemctl is-system-running -q > /dev/null 2>&1 && IS_SYSTEMD=1 || true

if [ -n "${IS_SYSTEMD}" -a -n "${IS_WSL2}" ];then
  if [ $(systemd-detect-virt) = 'wsl' ];then
     WSL2_SYSTEMD=1
  fi
fi

if [ -n "$WSL2_SYSTEMD" ];then
  print_info "k8s node run on WSL2(systemd)"

  source wsl2/.env
  TMP_ROOT="${K8S_ROOT}/../.khs1994-docker-lnmp/caches"
elif [ -n "$IS_WSL2" ];then
  print_info "k8s node run on WSL2"
  source wsl2/.env

  TMP_ROOT="${K8S_ROOT}/../.khs1994-docker-lnmp/caches"
fi

$(_sudo) mkdir -p ${K8S_ROOT}/{bin,etc/kubernetes/pki,opt,usr,var}

$(_sudo) mkdir -p ${TMP_ROOT} || TMP_ROOT=/tmp

$(_sudo) chmod 777 $HOME/.khs1994-docker-lnmp || true

export PATH=${K8S_ROOT}/bin:$PATH

case `uname -m` in
  x86_64 )
  GOARCH=amd64
  ;;
  armv7l )
  GOARCH=arm
  ;;
  aarch64 )
  GOARCH=arm64
esac

GOOS=`uname -s | tr A-Z  a-z`

command -v curl > /dev/null || print_info "curl not install"

CODING_URL=https://pcit-generic.pkg.coding.net/khs1994-docker/kubernetes

_upload_to_coding(){
  if [ "$UPLOAD_TO_CODING" != 'true' ];then return; fi

  local PKG_PATH=$1
  local PKG=$2
  local PKG_VERSION=$3

  curl -T ${PKG_PATH} -u ${CODING_USERNAME}:${CODING_TOKEN} \
  "${CODING_URL}/${PKG}?version=${PKG_VERSION}"
}

_get_kubernetes_server_by_docker_image(){
  local TYPE=server
  test -n "${1}" && local GOOS=$1
  test -n "${2}" && local GOARCH=$2
  test -n "${3}" && local TYPE=$3
  test -n "${4}" && local KUBERNETES_VERSION=$4

  if [ -f kubernetes-release/release/kubernetes-${TYPE}-${GOOS}-${GOARCH}-${KUBERNETES_VERSION}.tar.gz ];then
     print_info "Already download"
     return
  fi

  if [ -z "${K8S_IMAGE_NS}" ];then
      K8S_IMAGE_NS=lwpm
  fi

  if [ -z "${K8S_IMAGE_REGISTRY}" ];then
    if [ "${LNMP_CN_ENV}" != 'false' ];then
      # 国内
      K8S_IMAGE_REGISTRY="ccr.ccs.tencentyun.com"
    # else
      # 国外
      # K8S_IMAGE_REGISTRY="useccr.ccs.tencentyun.com"
      # K8S_IMAGE_NS="khs1994"
    fi
  fi

  # docker manifest inspect ${K8S_IMAGE_REGISTRY:-"docker.io"}/${K8S_IMAGE_NS}/kubernetes-${TYPE}:${KUBERNETES_VERSION}
  # docker container rm khs1994-kubernetes-${TYPE} > /dev/null 2>&1 || true
  # docker container create --name khs1994-kubernetes-${TYPE} \
  # docker container ls -a --filter name=khs1994-kubernetes-${TYPE}
  # docker container cp khs1994-kubernetes-${TYPE}:/kubernetes-${TYPE}-${GOOS}-${GOARCH}.tar.gz kubernetes-release/release/kubernetes-${TYPE}-${GOOS}-${GOARCH}-${KUBERNETES_VERSION}.tar.gz
  # docker container rm khs1994-kubernetes-${TYPE}

  set -x

  local ScriptRoot="$( cd "$( dirname "$0"  )" && pwd  )"

  dist=$($ScriptRoot/../windows/sdk/dockerhub/rootfs.sh \
  ${K8S_IMAGE_NS}/kubernetes-${TYPE} \
  ${KUBERNETES_VERSION} \
  ${GOARCH} \
  ${GOOS} \
  "" 0 ${K8S_IMAGE_REGISTRY:-registry.hub.docker.com}
  )

  mv $dist \
    kubernetes-release/release/kubernetes-${TYPE}-${GOOS}-${GOARCH}-${KUBERNETES_VERSION}.tar.gz

  set +x
}

_get_kubernetes_server_by_url(){
  curl https://storage.googleapis.com/kubernetes-release/release/stable.txt > /dev/null \
    && ( \
      curl -A "curl" -L https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/kubernetes-${TYPE:-server}-${GOOS}-${GOARCH}.tar.gz \
        -o kubernetes-release/release/kubernetes-${TYPE:-server}-${GOOS}-${GOARCH}-${KUBERNETES_VERSION}.tar.gz \
    ) && return 0 || return 1
}

_kubernetes_server_install(){
  if [ "$1" = "--url" ];then
    local get_by_url=1
    shift 1
  fi

  if [ "$GOOS" = 'darwin' ];then
    local GOOS=linux
  fi

  test -n "${1}" && local GOOS=$1
  test -n "${2}" && local GOARCH=$2
  test -n "${3}" && local TYPE=$3
  test -n "${4}" && local KUBERNETES_VERSION=$4

  __untar(){
    test -n "${1}" && local GOOS=$1
    test -n "${2}" && local GOARCH=$2
    test -n "${3}" && local TYPE=$3
    test -n "${4}" && local KUBERNETES_VERSION=$4

    tar -zxvf \
      kubernetes-${TYPE:-server}-${GOOS}-${GOARCH}-${KUBERNETES_VERSION}.tar.gz \
      --exclude "kubernetes/LICENSES" \
      --exclude "kubernetes/server/bin/*.docker_tag" \
      --exclude "kubernetes/server/bin/*.tar" \
      --exclude "kubernetes/kubernetes-src.tar.gz"

    # pwd => kubernetes-release/release/
    mv kubernetes v${KUBERNETES_VERSION}-${GOOS}-${GOARCH}/
  }

print_info "install kubernetes ${KUBERNETES_VERSION}-${GOOS}-${GOARCH} ..."

if [ -d kubernetes-release/release/v${KUBERNETES_VERSION}-${GOOS}-${GOARCH}/kubernetes/server/bin ];then
  print_info "Kubernetes files existed"
  return
fi

mkdir -p kubernetes-release/release/v${KUBERNETES_VERSION}-${GOOS}-${GOARCH}

  if [ -n "$get_by_url" ];then
    _get_kubernetes_server_by_url $@
    cd kubernetes-release/release
    __untar $@

    return
  fi

  # test $GOOS != 'linux' && exit

  # command -v docker > /dev/null

  # if [ $? -eq 0 ];then
    # docker container ls > /dev/null 2>&1
    _get_kubernetes_server_by_docker_image $@
    cd kubernetes-release/release
    __untar $@
  # fi
}

kubectl_install(){
  print_info "install kubectl ..."

  command -v kubectl && return || true

  if [ "${LNMP_CN_ENV}" != 'false' -a "$OS" = 'Linux' -a "$GOARCH" = 'amd64' ];then
      local KUBECTL_VERSION=$(curl -fsSL https://mirror.azure.cn/kubernetes/kubectl/stable.txt)
      $(_sudo) $(_curl) -o /usr/local/bin/kubectl https://mirror.azure.cn/kubernetes/kubectl/${KUBECTL_VERSION}/bin/linux/amd64/kubectl
      $(_sudo) chmod +x /usr/local/bin/kubectl
      kubectl version
      return
  fi

  KUBECTL_VERSION=$(curl https://storage.googleapis.com/kubernetes-release/release/stable.txt)

  $(_curl) ${KUBECTL_URL}/${KUBECTL_VERSION}/bin/${GOOS}/${GOARCH}/kubectl \
      -o ${TMP_ROOT}/kubectl-${GOOS}-${GOARCH}

  $(_sudo) mv ${TMP_ROOT}/kubectl-${GOOS}-${GOARCH} /usr/local/bin/kubectl

  kubectl version
}

kubectl_info(){
  print_info "kubectl latest version is"

  local url=https://storage.googleapis.com/kubernetes-release/release/stable.txt
  if [ "${LNMP_CN_ENV}" != 'false' ];then
    url=https://mirror.azure.cn/kubernetes/kubectl/stable.txt
  fi
  $(_curl) $url
}

_nginx_build(){
  local url="https://nginx.org/download/"
  local TAR_FILE="nginx-${NGINX_VERSION}.tar.gz"

  if [ -f ${K8S_ROOT}/kube-nginx.tar.gz ];then
    print_info "${K8S_ROOT}/kube-nginx.tar.gz already exists"
    return
  fi

  print_info "build nginx ..."

  if [ -f ${TMP_ROOT}/${TAR_FILE} ];then
    true
  else
    rm -rf ${TMP_ROOT}/${TAR_FILE}
    $(_sudo) $(_curl) ${url}/${TAR_FILE} -o ${TMP_ROOT}/${TAR_FILE}
  fi

  $(_sudo) mkdir -p ${TMP_ROOT}/kube-nginx
  $(_sudo) tar -zxvf ${TMP_ROOT}/${TAR_FILE} -C ${TMP_ROOT}/kube-nginx
  cd ${TMP_ROOT}/kube-nginx/nginx-${NGINX_VERSION}

  $(_sudo) ./configure --prefix=/opt/k8s/kube-nginx \
    --with-stream --without-http \
    --without-http_uwsgi_module \
    --without-http_scgi_module --without-http_fastcgi_module

  $(_sudo) make
  $(_sudo) make install
  cd /opt/k8s/
  $(_sudo) tar -zcvf kube-nginx.tar.gz kube-nginx
  $(_sudo) cp kube-nginx.tar.gz ${TMP_ROOT}/
  cd $EXEC_CMD_DIR
  $(_sudo) rm -rf ${TMP_ROOT}/kube-nginx
}

_use_cn_mirror(){
if [ "${LNMP_CN_ENV}" = 'false' ];then
  return 1;
fi

local httpCode=`curl -X GET -sI ${1} | head -1 | awk '{print $2}'`

if [ "$httpCode" != 200 ];then
  echo 'cn mirror http response code is not 200, not use'
  return 1;
fi
}

_etcd_install(){
  local url_official="https://github.com/etcd-io/etcd/releases/download"
  local url="https://mirrors.huaweicloud.com/etcd"
  local TAR_FILE="etcd-v${ETCD_VERSION}-${GOOS}-${GOARCH}.tar.gz"

  if [ -f ${K8S_ROOT}/bin/etcd -a ! "$1" = "--dry-run" ];then
    print_info "etcd already install"

    ${K8S_ROOT}/bin/etcd --version || true
    return
  fi

  print_info "install etcd ..."

  if [ -f ${TMP_ROOT}/${TAR_FILE} ];then
    true
  else
    rm -rf ${TMP_ROOT}/${TAR_FILE}
    _use_cn_mirror ${url}/v${ETCD_VERSION}/${TAR_FILE} || url=$url_official

    $(_sudo) $(_curl) ${url}/v${ETCD_VERSION}/${TAR_FILE} -o ${TMP_ROOT}/${TAR_FILE}
  fi

  if [ "$1" = "--dry-run" ];then
    print_info "download etcd success"
    return
  fi

  $(_sudo) tar --no-same-owner -zxf ${TMP_ROOT}/${TAR_FILE} -C ${TMP_ROOT}
  $(_sudo) cp ${TMP_ROOT}/etcd-v${ETCD_VERSION}-${GOOS}-${GOARCH}/etcd* ${K8S_ROOT}/bin/
  $(_sudo) chown -R root:root ${K8S_ROOT}/bin/
  $(_sudo) rm -rf ${TMP_ROOT}/etcd-v${ETCD_VERSION}-${GOOS}-${GOARCH}
}

_helm_install(){
  if [ -f ${K8S_ROOT}/bin/helm -a ! "$1" = "--dry-run" ];then
    print_info "helm already install"

    ${K8S_ROOT}/bin/helm version || true
    return
  fi

  local url_official="https://get.helm.sh"
  local url="https://mirrors.huaweicloud.com/helm/v${HELM_VERSION}"
  local TAR_FILE="helm-v${HELM_VERSION}-${GOOS}-${GOARCH}.tar.gz"
  __download(){
    _use_cn_mirror ${url}/${TAR_FILE} || url=$url_official

    $(_sudo) $(_curl) ${url}/${TAR_FILE} -o ${TMP_ROOT}/${TAR_FILE}
  }

  print_info "install helm ..."

  if [ -f ${TMP_ROOT}/${TAR_FILE} ];then
      local sha256=$(sha256sum ${TMP_ROOT}/${TAR_FILE} | cut -d " " -f 1)
    if ! [ $sha256 = `curl -fsSL ${url}/${TAR_FILE}.sha256` ];then
      rm -rf ${TMP_ROOT}/${TAR_FILE}
      __download
    fi
  else
      __download
  fi

  if [ "$1" = "--dry-run" ];then
    print_info "download helm success"
    return
  fi

  $(_sudo) tar -zxf ${TMP_ROOT}/${TAR_FILE} -C ${TMP_ROOT}
  $(_sudo) cp ${TMP_ROOT}/${GOOS}-${GOARCH}/helm ${K8S_ROOT}/bin/
  $(_sudo) rm -rf ${TMP_ROOT}/${GOOS}-${GOARCH}
}

_gvisor_containerd_conf(){
  if ! [ "$GOOS" = 'linux' -a "$GOARCH" = 'amd64' ];then
    print_info "$GOOS $GOARCH not support gvisor, skip config containerd with runsc"

    return
  fi
print_info "config containerd with gvisor(runsc) ..."
# containerd 1.5.x
cat <<EOF | $(_sudo) tee -a ${K8S_ROOT}/etc/cri-containerd/1.5/config.toml
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc]
  runtime_type = "io.containerd.runsc.v1"
  # privileged_without_host_devices = false
  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc.options]
    SystemdCgroup = false
EOF
}

_gvisor_crio_conf(){
  print_info "config crio with gvisor(runsc) ..."
  cat <<EOF | $(_sudo) tee -a ${K8S_ROOT}/etc/crio/crio.conf
[crio.runtime.runtimes.runsc]
  runtime_path = "/usr/local/bin/runsc"
EOF
}

_gvisor_install(){
  if ! [ "$GOOS" = 'linux' -a "$GOARCH" = 'amd64' ];then
    print_info "$GOOS $GOARCH not support gvisor"

    return
  fi

  print_info "install gvisor(runsc) ..."
  local url_official="https://github.com/docker-practice/gvisor-mirror/releases/download/nightly"
  local url="https://download.fastgit.org/docker-practice/gvisor-mirror/releases/download/nightly"
  local TAR_FILE="runsc-${GOOS}-${GOARCH}.tar.gz"

  if [ "$DOWNLOAD_FROM_CODING" != "false" ];then
    url=${CODING_URL}/${TAR_FILE}?version=nightly
  else
    _use_cn_mirror ${url}/${TAR_FILE} && url=${url}/${TAR_FILE} \
                                      || url=${url_official}/${TAR_FILE}
  fi

  if [ "$1" = '-f' ];then $(_sudo) rm -rf ${TMP_ROOT}/${TAR_FILE}; fi

  if ! [ -f ${TMP_ROOT}/${TAR_FILE} ];then
    $(_sudo) $(_curl) ${url} -o ${TMP_ROOT}/${TAR_FILE}
  fi

  if [ "$1" = "--dry-run" ];then
    print_info "download gvisor(runsc) success"
    return
  fi

  $(_sudo) tar -zxf ${TMP_ROOT}/${TAR_FILE} -C /usr/local/bin

  runsc --version

  _upload_to_coding ${TMP_ROOT}/${TAR_FILE} ${TAR_FILE} nightly
}

_runsc_install(){
   _gvisor_install
}

_runc_install(){
  if [ `command -v runc` ];then
    print_info "runc already install"
    runc --version
    _upload_to_coding /usr/bin/runc runc.${GOARCH} ${RUNC_VERSION}
    return
  fi

  if ! [ "$GOARCH" = 'amd64' ];then
    # print_info "install runc ..."
    print_info "runc not found, please install docker-ce first"

    return
  fi

  local file_name=runc.${GOARCH}
  local url_official=https://github.com/opencontainers/runc/releases/download/v${RUNC_VERSION}
  local url=https://download.fastgit.org/opencontainers/runc/releases/download/v${RUNC_VERSION}

  if [ "$DOWNLOAD_FROM_CODING" != "false" ];then
    url=${CODING_URL}/${file_name}?version=${RUNC_VERSION}
  else
    _use_cn_mirror ${url}/${file_name} && url=${url}/${file_name} \
                                       || url=${url_official}/${file_name}
  fi

  $(_sudo) $(_curl) ${url} -o /usr/bin/runc

  $(_sudo) chmod +x /usr/bin/runc

  runc --version

  _upload_to_coding /usr/bin/runc runc.${GOARCH} ${RUNC_VERSION}
}

_crun_containerd_conf(){
  if ! [ "$GOOS" = 'linux' -a "$GOARCH" = 'amd64' ];then
    print_info "$GOOS $GOARCH not support crun, skip config containerd with crun"

    return
  fi
print_info "config containerd with crun ..."
# containerd 1.5.x
cat <<EOF | $(_sudo) tee -a ${K8S_ROOT}/etc/cri-containerd/1.5/config.toml
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.crun]
  runtime_type = "io.containerd.runc.v2"
  # privileged_without_host_devices = false
  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.crun.options]
    SystemdCgroup = true
    BinaryName = "crun"
EOF
}

_crun_crio_conf(){
  print_info "config crio with crun ..."
  cat <<EOF | $(_sudo) tee -a ${K8S_ROOT}/etc/crio/crio.conf
[crio.runtime.runtimes.crun]
  runtime_path = "/usr/local/bin/crun"
EOF
}

_crun_install(){
  if [ `command -v crun` ];then
    print_info "crun already install"
    crun --version
    _upload_to_coding /usr/bin/crun crun-${CRUN_VERSION}-linux-${GOARCH} ${CRUN_VERSION}
    return
  fi

  if ! [ "$GOARCH" = 'amd64' -o "$GOARCH" = 'arm64' ];then
    # print_info "install runc ..."
    print_info "Download crun only support amd64 and arm64"

    return
  fi

  local file_name=crun-${CRUN_VERSION}-linux-${GOARCH}
  local url=https://download.fastgit.org/containers/crun/releases/download/${CRUN_VERSION}
  local url_official=https://github.com/containers/crun/releases/download/${CRUN_VERSION}

  if [ "$DOWNLOAD_FROM_CODING" != "false" ];then
    url=${CODING_URL}/${file_name}?version=${CRUN_VERSION}
  else
    _use_cn_mirror ${url}/${file_name} && url=${url}/${file_name} \
                                       || url=${url_official}/${file_name}
  fi

  $(_sudo) $(_curl) ${url} -o /usr/bin/crun

  file_name=crun-${CRUN_VERSION}-linux-${GOARCH}-disable-systemd
  url=https://download.fastgit.org/containers/crun/releases/download/${CRUN_VERSION}
  url_official=https://github.com/containers/crun/releases/download/${CRUN_VERSION}

  if [ "$DOWNLOAD_FROM_CODING" != "false" ];then
    url=${CODING_URL}/${file_name}?version=${CRUN_VERSION}
  else
    _use_cn_mirror ${url}/${file_name} && url=${url}/${file_name} \
                                       || url=${url_official}/${file_name}
  fi

  $(_sudo) $(_curl) ${url} \
    -o /usr/bin/crun-disable-systemd

  $(_sudo) chmod +x /usr/bin/crun /usr/bin/crun-disable-systemd

  crun --version
  crun-disable-systemd --version

  _upload_to_coding /usr/bin/crun crun-${CRUN_VERSION}-linux-${GOARCH} ${CRUN_VERSION}
  _upload_to_coding /usr/bin/crun-disable-systemd crun-${CRUN_VERSION}-linux-${GOARCH}-disable-systemd ${CRUN_VERSION}
}

_containerd_install(){
  if [ "$1" = "-f" ];then $(_sudo) rm -rf ${K8S_ROOT}/bin/cri-containerd;fi

  if ! [ "$GOARCH" = 'amd64' -o "$GOARCH" = 'arm64' ];then
    echo "only support [ amd64 ] and [ arm64 ]"

    return
  fi

  if [ -f ${K8S_ROOT}/bin/cri-containerd -a ! "$1" = "--dry-run" ];then \
    print_info "containerd 1.5 already install"; \
    ${K8S_ROOT}/bin/cri-containerd --version;return; fi

  print_info "install containerd 1.5 ..."

  if [ "$GOOS" = "linux" -a "$GOARCH" = 'arm64' ];then
    local url_official="https://github.com/docker-practice/containerd/releases/download/nightly"
    local url="https://download.fastgit.org/docker-practice/containerd/releases/download/nightly"
    CONTAINERD_VERSION="nightly"
  else
    local url_official="https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERSION}"
    local url="https://download.fastgit.org/containerd/containerd/releases/download/v${CONTAINERD_VERSION}"
  fi

  local TAR_FILE="containerd-${CONTAINERD_VERSION}-${GOOS}-${GOARCH}.tar.gz"

  if [ "$DOWNLOAD_FROM_CODING" != "false" ];then
    url=${CODING_URL}/${TAR_FILE}?version=${CONTAINERD_VERSION}
    local sha256sum_url=${CODING_URL}/${TAR_FILE}.sha256sum?version=${CONTAINERD_VERSION}
  else
    _use_cn_mirror ${url}/${TAR_FILE} && url=${url}/${TAR_FILE} \
                                      || url=${url_official}/${TAR_FILE}

    local sha256sum_url=${url}.sha256sum
  fi

  if [ -f ${TMP_ROOT}/${TAR_FILE} ];then
    local sha256=$(sha256sum ${TMP_ROOT}/${TAR_FILE} | cut -d " " -f 1)
    local sha256_remote=`curl -fsSL ${sha256sum_url} | cut -d " " -f 1` || true

    if [ $sha256 != "$sha256_remote" -a "$GOARCH" = 'amd64' ];then
      $(_sudo) rm -rf ${TMP_ROOT}/${TAR_FILE}
      $(_sudo) $(_curl) ${url} -o ${TMP_ROOT}/${TAR_FILE}
    fi
  else
      $(_sudo) $(_curl) ${url} -o ${TMP_ROOT}/${TAR_FILE}
  fi

  if [ "$1" = "--dry-run" ];then
    print_info "download containerd success"
    return
  fi

  $(_sudo) tar -zxf ${TMP_ROOT}/${TAR_FILE} -C ${K8S_ROOT}

  $(_sudo) install -m755 ${K8S_ROOT}/bin/containerd ${K8S_ROOT}/bin/cri-containerd
  $(_sudo) install -m755 ${K8S_ROOT}/bin/containerd-shim /usr/local/bin/cri-containerd-shim
  $(_sudo) install -m755 ${K8S_ROOT}/bin/containerd-shim-runc-v1 /usr/local/bin/
  $(_sudo) install -m755 ${K8S_ROOT}/bin/containerd-shim-runc-v2 /usr/local/bin/

  _upload_to_coding ${TMP_ROOT}/${TAR_FILE} ${TAR_FILE} ${CONTAINERD_VERSION}
  $(_sudo) $(_curl) ${sha256sum_url} -o ${TMP_ROOT}/${TAR_FILE}.sha256sum
  _upload_to_coding ${TMP_ROOT}/${TAR_FILE}.sha256sum ${TAR_FILE}.sha256sum ${CONTAINERD_VERSION}
}

_cni_install(){
  $(_sudo) mkdir -p ${K8S_ROOT}/opt/cni/bin
  if [ "$1" = "-f" ];then $(_sudo) rm -rf ${K8S_ROOT}/opt/cni/bin/loopback;fi
  if [ -f ${K8S_ROOT}/opt/cni/bin/loopback -a ! "$1" = "--dry-run" ];then
    print_info "cni already install"
    return
  fi

  local url="https://mirror.azure.cn/kubernetes/containernetworking-plugins"
  local url_official="https://github.com/containernetworking/plugins/releases/download/v${CNI_VERSION}"
  local TAR_FILE="cni-plugins-${GOOS}-${GOARCH}-v${CNI_VERSION}.tgz"

  # wsl2 with cni 0.8.2 error:
  # https://github.com/containernetworking/plugins/issues/370
  # down to 0.7.6
  # 0.8.3 fixed

  _use_cn_mirror ${url}/${TAR_FILE} || url=$url_official

  __download(){
    $(_sudo) $(_curl) ${url}/${TAR_FILE} -o ${TMP_ROOT}/${TAR_FILE}
  }

  print_info "install cni ..."

  if [ -f ${TMP_ROOT}/${TAR_FILE} ];then
      local sha256=$(sha256sum ${TMP_ROOT}/${TAR_FILE} | cut -d " " -f 1)
    if ! [ $sha256 = `curl -fsSL ${url}/${TAR_FILE}.sha256 | cut -d " " -f 1` ];then
      $(_sudo) rm -rf ${TMP_ROOT}/${TAR_FILE}
      __download
    fi
  else
      __download
  fi

  if [ "$1" = "--dry-run" ];then
    print_info "download cni success"
    return
  fi

  $(_sudo) tar -zxf ${TMP_ROOT}/${TAR_FILE} -C ${K8S_ROOT}/opt/cni/bin/
}

_crictl_docker_conf(){
  $(_sudo) cp systemd/etc/crictl.yaml /etc/crictl.yaml
}

_crictl_containerd_conf(){
  $(_sudo) rm -rf /etc/crictl.yaml
  print_info "conf crictl with containerd"
  cat <<EOF | $(_sudo) tee -a /etc/crictl.yaml
runtime-endpoint: unix:///run/cri-containerd/cri-containerd.sock
image-endpoint: unix:///run/cri-containerd/cri-containerd.sock
timeout: 10
debug: false
EOF
}

_crictl_crio_conf(){
  $(_sudo) rm -rf /etc/crictl.yaml
  print_info "conf crictl with cri-o"
  cat <<EOF | $(_sudo) tee -a /etc/crictl.yaml
runtime-endpoint: unix:///var/run/crio/crio.sock
image-endpoint: unix:///var/run/crio/crio.sock
timeout: 10
debug: false
EOF
}

_crictl_install(){
  if [ "$1" = "-f" ];then $(_sudo) rm -rf /usr/bin/crictl;fi
  if [ -f /usr/bin/crictl -a ! "$1" = "--dry-run" ];then
    print_info "crictl already install"
    crictl --version
    return
  fi

  local url_official="https://github.com/kubernetes-sigs/cri-tools/releases/download/v${CRICTL_VERSION}"
  local url="https://download.fastgit.org/kubernetes-sigs/cri-tools/releases/download/v${CRICTL_VERSION}"
  local TAR_FILE="crictl-v${CRICTL_VERSION}-linux-${GOARCH}.tar.gz"

  if [ "$DOWNLOAD_FROM_CODING" != "false" ];then
    url=${CODING_URL}/${TAR_FILE}?version=${CRICTL_VERSION}
  else
    _use_cn_mirror ${url}/${TAR_FILE} && url=${url}/${TAR_FILE} \
                                      || url=${url_official}/${TAR_FILE}
  fi

  print_info "install crictl ..."

  # if [ -f ${TMP_ROOT}/${TAR_FILE} ];then
    # local sha256=$(sha256sum ${TMP_ROOT}/${TAR_FILE} | cut -d " " -f 1)
    # if ! [ $sha256 = "876dd2b3d0d1c2590371f940fb1bf1fbd5f15aebfbe456703ee465d959700f4a" ];then
    #   if [ $GOARCH = 'amd64' ];then
    #     $(_sudo) rm -rf ${TMP_ROOT}/${TAR_FILE}
    #     __download
    #   fi
    # fi
  # else
  if ! [ -f ${TMP_ROOT}/${TAR_FILE} ];then
      $(_sudo) $(_curl) ${url} -o ${TMP_ROOT}/${TAR_FILE}
  fi

  if [ "$1" = "--dry-run" ];then
    print_info "download crictl success"
    return
  fi

  # $(_sudo) tar -zxf ${TMP_ROOT}/${TAR_FILE} -C ${K8S_ROOT}/bin/
  $(_sudo) tar -zxf ${TMP_ROOT}/${TAR_FILE} -C /usr/bin/

  _upload_to_coding ${TMP_ROOT}/${TAR_FILE} ${TAR_FILE} ${CRICTL_VERSION}
}

_k8s_install_dep(){
  print_info "install dep ..."
  # 安装依赖软件
  command -v dnf > /dev/null && ($(_sudo) dnf install -y conntrack ipvsadm ipset jq iptables curl socat \
    sysstat libseccomp > /dev/null) || command -v yum > /dev/null && ($(_sudo) yum install -y conntrack ipvsadm ipset jq iptables curl socat unzip \
    sysstat libseccomp > /dev/null) && return

  command -v apt > /dev/null && $(_sudo) apt update -qq ; $(_sudo) apt install -y conntrack ipvsadm ipset jq iptables curl socat unzip \
    sysstat libseccomp2 kmod > /dev/null || true
}

_k8s_install_disable_firewalld(){
  # 关闭防火墙
  print_info "stop and disable firewalld"
  $(_sudo) systemctl stop firewalld && $(_sudo) systemctl disable firewalld || true
}

_k8s_install_cp_bin(){
  if [ ! -f ${K8S_ROOT}/bin/kubeadm -o "$1" = '-f' ];then
    _kubernetes_server_install
    cd $EXEC_CMD_DIR > /dev/null
    $(_sudo) cp $PWD/kubernetes-release/release/v${KUBERNETES_VERSION}-${GOOS}-${GOARCH}/kubernetes/server/bin/* ${K8S_ROOT}/bin/
  fi

  $(_sudo) install -m755 bin/*.sh ${K8S_ROOT}/bin/

  $(_sudo) chmod +x ${K8S_ROOT}/bin/*
}

_k8s_install_systemd_docker(){
  print_info "Update systemd Docker unit file ..."
  $(_sudo) mkdir -p /etc/systemd/system/docker.service.d
  $(_sudo) cp systemd/system/docker-10-cri-docker.conf /etc/systemd/system/docker.service.d/

  $(_sudo) cp systemd/system/kubelet-20-cri-docker.conf /etc/systemd/system/kubelet.service.d/20-cri-docker.conf
  $(_sudo) sed -i "s#/opt/k8s#${K8S_ROOT}#g" /etc/systemd/system/kubelet.service.d/20-cri-docker.conf
}

_k8s_install_kubectl_conf(){
  mkdir -p ~/.kube

  if ! [ -f ~/.kube/config ];then
    cp systemd/certs/kubectl.kubeconfig ~/.kube/config
  fi
}

_cni_config(){
  print_info "config cni ..."
  $(_sudo) mkdir -p ${K8S_ROOT}/etc/cni/net.d
  $(_sudo) cp systemd/etc/cni.example/99-loopback.conf ${K8S_ROOT}/etc/cni/net.d
  # $(_sudo) cp systemd/etc/cni.example/10-mynet.conf ${K8S_ROOT}/etc/cni/net.d
}

_cri_sysctl(){
  print_info "exec sysctl to meet cri..."
  $(_sudo) rm -rf /etc/sysctl.d/99-kubernetes-cri.conf /etc/sysctl.d/99-kubernetes-calico.conf
  $(_sudo) touch /etc/sysctl.d/99-kubernetes-cri.conf
  $(_sudo) touch /etc/sysctl.d/99-kubernetes-calico.conf
  echo "net.bridge.bridge-nf-call-iptables  = 1" | $(_sudo) tee -a /etc/sysctl.d/99-kubernetes-cri.conf > /dev/null
  echo "net.ipv4.ip_forward                 = 1" | $(_sudo) tee -a /etc/sysctl.d/99-kubernetes-cri.conf > /dev/null
  echo "net.bridge.bridge-nf-call-ip6tables = 1" | $(_sudo) tee -a /etc/sysctl.d/99-kubernetes-cri.conf > /dev/null
  # calico
  echo "net.ipv4.conf.all.rp_filter = 1" | $(_sudo) tee -a /etc/sysctl.d/99-kubernetes-calico.conf
  $(_sudo) sysctl --system || true
}

_conmon_install(){
  if [ -f ${K8S_ROOT}/usr/local/bin/conmon ];then
    return
  fi

  local url=https://github.com/containers/conmon/releases/download/v${CONMON_VERSION}/conmon.$GOARCH
  local url_official=https://download.fastgit.org/containers/conmon/releases/download/v${CONMON_VERSION}/conmon.$GOARCH

  if [ $DOWNLOAD_FROM_CODING != "false" ];then
    url=$CODING_URL/conmon.$GOARCH?version=${CONMON_VERSION}
  else
    _use_cn_mirror $url || url=$url_official
  fi

  if [ $OS = "Linux" ];then
    if [ $GOARCH = "amd64" -o $GOARCH = "arm64" ];then
      $(_sudo) $(_curl) -o ${K8S_ROOT}/usr/local/bin/conmon $url

      $(_sudo) chmod +x ${K8S_ROOT}/usr/local/bin/conmon

      _upload_to_coding ${K8S_ROOT}/usr/local/bin/conmon conmon.$GOARCH ${CONMON_VERSION}

      return
    fi
  fi

  cd /tmp

  if [ -d conmon/.git ];then
    cd conmon
    git fetch origin master --depth=1
    git reset --hard origin/master
  else
    rm -rf conmon
    git clone --depth=1 https://github.com/containers/conmon
  fi

  cd /tmp/conmon
  make
  $(_sudo) make PREFIX=${K8S_ROOT}/usr/local install
}

# https://github.com/cri-o/cri-o/blob/master/install.md#build-and-install-cri-o-from-source
_crio_install(){
  set +e
  command -v crio > /dev/null 2>&1
  if [ $? -eq 0 ];then print_info "cri-o already install" ; return;fi
  set -e

  print_info "install cri-o ..."

  local CRIO_VERSION_X=`echo $KUBERNETES_VERSION | cut -d "." -f 1`
  local CRIO_VERSION_Y=`echo $KUBERNETES_VERSION | cut -d "." -f 2`
  local CRIO_VERSION="$CRIO_VERSION_X"."$CRIO_VERSION_Y"

  set -x

  command -v dnf > /dev/null && isRPM=1 || true
  command -v yum > /dev/null && isRPM=1 || true

  if [ -n "$isRPM" ];then $(_sudo) yum install -y \
  git \
  glib2-devel \
  glibc-devel \
  glibc-static \
  $(command -v go > /dev/null || echo "go") \
  gpgme-devel \
  libassuan-devel \
  libgpg-error-devel \
  libseccomp-devel \
  libselinux-devel \
  pkgconfig \
  make; fi

  # $(_sudo) sed -i "s#deb.debian.org#mirrors.ustc.edu.cn#g" /etc/apt/sources.list
  # $(_sudo) sed -i "s#security.debian.org#mirrors.ustc.edu.cn#g" /etc/apt/sources.list
  #
  # $(_sudo) sed -i "s#archive.ubuntu.com#mirrors.ustc.edu.cn#g" /etc/apt/sources.list
  # $(_sudo) sed -i "s#security.ubuntu.com#mirrors.ustc.edu.cn#g" /etc/apt/sources.list

  # && . /etc/os-release \
  # && sudo sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/x${NAME}_${VERSION_ID}/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" \
  # && wget -nv https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/x${NAME}_${VERSION_ID}/Release.key -O- | sudo apt-key add - \
  # && $(_sudo) apt-get update -qq \

  command -v apt > /dev/null && isAPT=1 || true
  if [ -n "$isAPT" ];then $(_sudo) apt update -qq || true;fi
  if [ -n "$isAPT" ];then \
  $(_sudo) apt install -y software-properties-common \
  && $(_sudo) apt-get install -y \
  git \
  $(command -v go > /dev/null || echo "golang-go") \
  libassuan-dev \
  libglib2.0-dev \
  libc6-dev \
  libgpgme11-dev \
  libgpg-error-dev \
  libseccomp-dev \
  libsystemd-dev \
  libselinux1-dev \
  pkg-config \
  libudev-dev \
  gcc \
  make; fi

  # containers-common
  $(_sudo) cp -r systemd/etc/containers/. ${K8S_ROOT}/etc/containers

  go_path=`go env GOPATH | cut -d ":" -f 1`

  export GOPATH=$go_path

  CRIO_GIT=https://github.com/cri-o/cri-o
  if ! [ "${LNMP_CN_ENV}" = "false" ];then
      CRIO_GIT=https://github.com.cnpmjs.org/cri-o/cri-o
  fi

  git clone --depth=1 -b release-${CRIO_VERSION} ${CRIO_GIT} $go_path/src/github.com/cri-o/cri-o \
    || git clone --depth=1 -b master ${CRIO_GIT} $go_path/src/github.com/cri-o/cri-o || true

  if ! [ -d $go_path/src/github.com/cri-o/cri-o ];then
    exit 1
  fi

  cd $go_path/src/github.com/cri-o/cri-o

  go version
  $(_sudo) go version || true
  go env
  $(_sudo) go env || true

  if ! [ "${LNMP_CN_ENV}" = "false" ];then
    $(_sudo) sed -i \
        's/proxy.golang.org/goproxy.cn,direct/g' \
        Makefile
  fi

  make clean
  make DESTDIR="${K8S_ROOT}" BUILDTAGS='seccomp exclude_graphdriver_btrfs exclude_graphdriver_devicemapper'
  # make build-static CONTAINER_RUNTIME=docker
  set +e
  command -v sudo > /dev/null 2>&1
  if [ $? = 0 ];then
    sudo PATH=$GOPATH/bin:$PATH GOPATH=$go_path \
      make DESTDIR="${K8S_ROOT}" install.bin
    sudo PATH=$GOPATH/bin:$PATH GOPATH=$go_path \
      make DESTDIR="${K8S_ROOT}" install.config
    sudo PATH=$GOPATH/bin:$PATH GOPATH=$go_path \
      make DESTDIR="${K8S_ROOT}" install.systemd
  else
    PATH=$GOPATH/bin:$PATH GOPATH=$go_path \
      make DESTDIR="${K8S_ROOT}" install.bin
    PATH=$GOPATH/bin:$PATH GOPATH=$go_path \
      make DESTDIR="${K8S_ROOT}" install.config
    PATH=$GOPATH/bin:$PATH GOPATH=$go_path \
      make DESTDIR="${K8S_ROOT}" install.systemd
  fi
  set -e

  _conmon_install
  # command -v yum > /dev/null && $(_sudo) yum install cri-o
  #
  # command -v apt > /dev/null && ($(_sudo) apt-add-repository -y ppa:projectatomic/ppa \
  #   && $(_sudo) apt-get update -qq \
  #   && $(_sudo) apt-get install cri-o-$CRIO_VERSION)
  set +x
}

_crio_config(){
  cd $EXEC_CMD_DIR
  print_info "config cri-o ..."
  # crio --config="" config --default > crio.conf
  sed "s!/opt/k8s!${K8S_ROOT}!g" systemd/etc/crio/crio.conf \
    | $(_sudo) tee ${K8S_ROOT}/etc/crio/crio.conf

  $(_sudo) sed -i "s!/opt/k8s!${K8S_ROOT}!g" \
    ${K8S_ROOT}/etc/containers/containers.conf
  $(_sudo) sed -i "s!/opt/k8s!${K8S_ROOT}!g" \
    ${K8S_ROOT}/etc/containers/storage.conf
  $(_sudo) sed -i "s!/opt/k8s!${K8S_ROOT}!g" \
    ${K8S_ROOT}/etc/containers/policy.json
  $(_sudo) sed -i "s!/opt/k8s!${K8S_ROOT}!g" \
    ${K8S_ROOT}/etc/containers/registries.d/default.yaml

  _cri_sysctl
}

# https://kubernetes.io/docs/setup/production-environment/container-runtimes/#containerd
_containerd_cri_config(){
  print_info "config containerd ..."
  $(_sudo) mkdir -p ${K8S_ROOT}/etc/cri-containerd/1.5
  # containerd config default > ${K8S_ROOT}/etc/cri-containerd/config.toml
  sed "s!/opt/k8s!${K8S_ROOT}!g" systemd/etc/cri-containerd/1.5/config.toml \
    | $(_sudo) tee ${K8S_ROOT}/etc/cri-containerd/1.5/config.toml

  _cri_sysctl
}

_docker_default_conf(){
  print_info "restore docker default conf"
  $(_sudo) rm -rf /etc/systemd/system/docker.service.d/10-cri-docker.conf
}

# k8s install

_check_master_ip(){
  if [ "${MASTER_IP}" = '192.168.199.100' ];then
    clear
    echo ""
    echo "======================================================"
    echo "**** you must edit MASTER_IP in systemd/.env file ****"
    echo "======================================================"
    echo ""
  fi
}

_cri_handler(){
  local CONTAINER_RUNTIME="--containerd"

  if [ -n "$1" ];then
    local CONTAINER_RUNTIME=$1
  fi

  $(_sudo) rm -rf /etc/systemd/system/kubelet.service.d
  $(_sudo) mkdir -p /etc/systemd/system/kubelet.service.d
  if [ "$CONTAINER_RUNTIME" = "--containerd" ];then
    print_info "Kubelet Runtime is Containerd"
    _containerd_install
    _containerd_cri_config
    _docker_default_conf
    if [ "$GOOS" = 'linux' ];then _gvisor_containerd_conf;fi
    _crictl_containerd_conf
  elif [ "$CONTAINER_RUNTIME" = "--crio" ];then
    print_info "Kubelet Runtime is CRI-O"
    _crio_install
    _crio_config
    _docker_default_conf
    if [ "$GOOS" = 'linux' ];then _gvisor_crio_conf;fi
    _crictl_crio_conf
  elif [ "$CONTAINER_RUNTIME" = "--docker" ];then
    print_info "Kubelet Runtime is Docker"
    sleep 15
    _k8s_install_systemd_docker
    _crictl_docker_conf
  else
    print_info "CRI only support containerd | crio | docker"
    exit 1
  fi
}

_k8s_install_conf_cp(){
  print_info "copy kubernetes conf and cert file ..."

  $(_sudo) cp systemd/certs/*.pem systemd/certs/sa.pub systemd/certs/sa.key ${K8S_ROOT}/etc/kubernetes/pki

  if ! [ -f systemd/certs/admin-key.pem ];then \
    echo "Please Generate certs first $ docker-compose up cfssl-local"; exit; fi

  $(_sudo) cp systemd/certs/*.kubeconfig ${K8S_ROOT}/etc/kubernetes/
  $(_sudo) cp \
    systemd/certs/{audit-policy.yaml,\
kubelet.config.yaml,\
kube-proxy.config.yaml,\
csr-crb.yaml,\
kube-scheduler.config.yaml,\
encryption-config.yaml} \
    ${K8S_ROOT}/etc/kubernetes/
}

# https://github.com/kubernetes/kubeadm/issues/273
# 16.04 not use systemd-resolved
# 18.04 use systemd-resolved

_replace_kubelet_resolv_conf(){
  local REPLACE_KUBELET_RESOLV_CONF=true
  $(_sudo) systemctl is-active systemd-resolved > /dev/null \
    || local REPLACE_KUBELET_RESOLV_CONF=false

  if [ $REPLACE_KUBELET_RESOLV_CONF = 'true' ];then
    print_info "replace_kubelet_resolv_conf"
    $(_sudo) sed -i "s#^resolvConf.*#resolvConf: /run/systemd/resolve/resolv.conf#g" \
    ${K8S_ROOT}/etc/kubernetes/kubelet.config.yaml
  fi

  $(_sudo) cat ${K8S_ROOT}/etc/kubernetes/kubelet.config.yaml | grep resolvConf
}

_k8s_install_conf_replace(){
  local files=("kube-proxy.config.yaml" "kubelet.config.yaml" "kube-scheduler.config.yaml")

  for file in ${files[@]}; \
  do \
    $(_sudo) sed -i "s!##NODE_IP##!${MASTER_IP}!g" \
      ${K8S_ROOT}/etc/kubernetes/$file ; \
    $(_sudo) sed -i "s#/opt/k8s#${K8S_ROOT}#g" \
      ${K8S_ROOT}/etc/kubernetes/$file ; \
    $(_sudo) sed -i "s!##NODE_NAME##!node1!g" \
      ${K8S_ROOT}/etc/kubernetes/$file ; \
  done
}

_enable_wsl_k8s_systemd_target(){
  $(_sudo) ln -sf /etc/systemd/system/$1 \
    /etc/systemd/system/wsl-k8s.target.wants/${2:-$1}
}

_handle_etcd_not_support_arch(){
  # https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/supported-platform.md
  if [ "${GOARCH}" = 'arm64' ];then
    $(_sudo) sed -i -e "s/_ETCD_UNSUPPORTED_ARCH/ETCD_UNSUPPORTED_ARCH/g" \
        -e "s/__GOARCH__/${GOARCH}/g" \
        /etc/systemd/system/etcd.service
  fi
}

_k8s_install_systemd(){
  print_info "copy systemd unit file $@ ... "

  if [ -n "${1}" ];then
    for item in $@
    do
      $(_sudo) cp systemd/system/${item}.service /etc/systemd/system/
    done
  else
    $(_sudo) cp systemd/system/*.service /etc/systemd/system/
  fi

  cd systemd/system

  for file in `ls *.service`; \
  do
    if [ ! -f /etc/systemd/system/$file ];then continue ; fi
    $(_sudo) sed -i "s#/opt/k8s#${K8S_ROOT}#g" \
      /etc/systemd/system/$file

    if [ -z "$WSL2_SYSTEMD" ];then
    $(_sudo) sed -i "s/192.168.199.100/${MASTER_IP}/g" \
      /etc/systemd/system/$file
    fi

    if [ -n "$IS_WSL2" ];then
      $(_sudo) sed -i "s!##NODE_NAME##!wsl2!g" \
        /etc/systemd/system/kubelet.service \
        /etc/systemd/system/kubelet@.service
    else
      $(_sudo) sed -i "s!##NODE_NAME##!${KUBE_HOSTNAME:-node1}!g" \
        /etc/systemd/system/kubelet.service \
        /etc/systemd/system/kubelet@.service
    fi

    $(_sudo) sed -i "s!##KUBE_APISERVER##!${KUBE_APISERVER}!g" \
      /etc/systemd/system/$file
  done

  if [ -n "$WSL2_SYSTEMD" ];then
    _etcd_install

    $(_sudo) cp ../../wsl2/conf/kube-scheduler.config.yaml.temp ${K8S_ROOT}/etc/kubernetes

    $(_sudo) cp ../../wsl2/conf/kube-proxy.config.yaml.temp ${K8S_ROOT}/etc/kubernetes
    $(_sudo) cp ../../wsl2/conf/kubelet.config.yaml.temp ${K8S_ROOT}/etc/kubernetes

    $(_sudo) sed -i \
      -e 's/192.168.199.100/%i/g' \
      -e 's/# ConditionVirtualization/ConditionVirtualization/g' \
      -e 's/# ConditionPathIsMountPoint/ConditionPathIsMountPoint/g' \
      -e 's/# BindsTo/BindsTo/g' \
      -e 's/# RequiresMountsFor/RequiresMountsFor/g' \
      -e 's/_K8S_ETCD_HOST/K8S_ETCD_HOST/g' \
      /etc/systemd/system/kube-apiserver.service

    $(_sudo) sed -i \
      -e 's/192.168.199.100/%i/g' \
      -e 's/__NODE_NAME__/wsl2/g' \
      -e 's/# ConditionVirtualization/ConditionVirtualization/g' \
      -e 's/# ConditionPathIsMountPoint/ConditionPathIsMountPoint/g' \
      -e 's/# BindsTo/BindsTo/g' \
      -e 's/# RequiresMountsFor/RequiresMountsFor/g' \
      /etc/systemd/system/kube-controller-manager.service \
      /etc/systemd/system/kube-scheduler.service \
      /etc/systemd/system/kube-proxy.service

    $(_sudo) sed -i 's/SystemdCgroup = true/SystemdCgroup = false/g' \
      /wsl/wsl-k8s-data/k8s/etc/cri-containerd/1.5/config.toml

    $(_sudo) sed -i \
      -e 's/# ConditionVirtualization/ConditionVirtualization/g' \
      -e 's/# ConditionPathIsMountPoint/ConditionPathIsMountPoint/g' \
      -e 's/# BindsTo/BindsTo/g' \
      -e 's/# RequiresMountsFor/RequiresMountsFor/g' \
      /etc/systemd/system/cri-containerd@.service \
      /etc/systemd/system/kubelet@.service \
      /etc/systemd/system/etcd.service

    $(_sudo) sed -i \
      's/${KUBE_APISERVER_HOST}/${WSL2_IP}/g' \
      /etc/systemd/system/kube-apiserver.service

    $(_sudo) sed -i \
      's/${K8S_CM_HOST}/${WSL2_IP}/g' \
      /etc/systemd/system/kube-controller-manager.service

    $(_sudo) sed -i \
      's/${K8S_S_HOST}/${WSL2_IP}/g' \
      /etc/systemd/system/kube-scheduler.service

    $(_sudo) sed -i \
      's/${K8S_ETCD_HOST}/127.0.0.1/g' \
      /etc/systemd/system/etcd.service

    $(_sudo) mkdir -p /etc/systemd/system/wsl-k8s.target.wants

    $(_sudo) cp target/wsl.target /etc/systemd/system/
    $(_sudo) cp target/wsl-k8s.target /etc/systemd/system/
    $(_sudo) cp target/wsl-k8s-shutdown.target /etc/systemd/system/

    _enable_wsl_k8s_systemd_target kube-apiserver.service
    _enable_wsl_k8s_systemd_target kube-controller-manager.service
    _enable_wsl_k8s_systemd_target kube-scheduler.service
    _enable_wsl_k8s_systemd_target cri-containerd@.service cri-containerd@1.5.service
    _enable_wsl_k8s_systemd_target kubelet@.service kubelet@cri-containerd.service
    _enable_wsl_k8s_systemd_target etcd.service

    $(_sudo) mkdir -p /etc/systemd/system/kube-.service.d \
                      /etc/systemd/system/cri-.service.d \
                      /etc/systemd/system/etcd.service.d \
                      /etc/systemd/system/kubelet@.service.d
    {
       echo "[Unit]";
       echo "ConditionVirtualization=wsl";
       echo "ConditionPathIsMountPoint=/wsl/wsl-k8s-data";
       echo "RequiresMountsFor=/wsl/wsl-k8s-data";
    } | $(_sudo) tee /etc/systemd/system/kube-.service.d/99-must-run-in-wsl2.conf \
                     /etc/systemd/system/cri-.service.d/99-must-run-in-wsl2.conf \
                     /etc/systemd/system/etcd.service.d/99-must-run-in-wsl2.conf \
                     /etc/systemd/system/kubelet@.service.d/99-must-run-in-wsl2.conf \
                     > /dev/null

    # $(_sudo) mkdir -p /etc/systemd/system/cri-.service.d
    # {
    #    echo "[Unit]";
    # } | $(_sudo) tee /etc/systemd/system/cri-.service.d/99-must-run-in-wsl2.conf \
    #                  > /dev/null

    $(_sudo) mkdir -p ${K8S_ROOT}/etc/systemd/system/kube-.service.d

    $(_sudo) cp -r kube-.service.d/. ${K8S_ROOT}/etc/systemd/system/kube-.service.d/
  fi # wsl2 end

  $(_sudo) cp -r mount/. /etc/systemd/system/

  $(_sudo) sed -i \
  "s!/opt/k8s!${K8S_ROOT}!g" \
  $(for item in $(cd mount && ls);do echo /etc/systemd/system/$item;done)

  $(_sudo) sed -i \
  's!__opt/k8s__!/opt/k8s!g' \
  /etc/systemd/system/kubelet@.service

  cd - > /dev/null

  _handle_etcd_not_support_arch

  $(_sudo) systemctl daemon-reload || true
}

_k8s_install_mkdir(){
  $(_sudo) mkdir --parents ${K8S_ROOT}/var/log/kubernetes
  $(_sudo) mkdir --parents ${K8S_ROOT}/var/lib/{kubelet,kube-proxy}
}

_k8s_install(){
  _check_master_ip

  if ! [ -f systemd/certs/ca.pem ];then
    print_info "please generate certs first"
    exit 1
  fi

  _k8s_install_dep

  $(_sudo) mkdir -p /etc/{docker,systemd/system}
  $(_sudo) chmod -R 755 /opt

  _k8s_install_disable_firewalld

  _etcd_install

  _cni_install
  _cni_config

  _helm_install

  _k8s_install_cp_bin

  _k8s_install_conf_cp

  _k8s_install_conf_replace
  _replace_kubelet_resolv_conf

  _gvisor_install

  _crictl_install
  _runc_install

  _cri_handler $1

  _k8s_install_systemd

  _k8s_install_mkdir

  # kubectl
  _k8s_install_kubectl_conf
}

# k8s join

_k8s_join_conf_cp(){
  print_info "copy kubernetes conf and cert file ..."
  $(_sudo) cp systemd/certs/{ca.pem,\
ca-key.pem} \
${K8S_ROOT}/etc/kubernetes/pki

  $(_sudo) cp \
    systemd/certs/{kubelet.config.yaml,\
kube-proxy.config.yaml,\
csr-crb.yaml,\
kubectl.kubeconfig,\
kube-proxy.kubeconfig} \
    ${K8S_ROOT}/etc/kubernetes
}

_k8s_join_conf_replace(){
  local files=("kube-proxy.config.yaml" "kubelet.config.yaml")

  for file in ${files[@]}; \
  do \
    $(_sudo) sed -i "s!##NODE_IP##!${NODE_IP}!g" \
      ${K8S_ROOT}/etc/kubernetes/$file ; \
    $(_sudo) sed -i "s#/opt/k8s#${K8S_ROOT}#g" \
      ${K8S_ROOT}/etc/kubernetes/$file ; \
    $(_sudo) sed -i "s!##NODE_NAME##!${KUBE_HOSTNAME}!g" \
      ${K8S_ROOT}/etc/kubernetes/$file ; \
  done
}

_k8s_join_hostname_handler(){
  KUBE_HOSTNAME=`hostname`

  if [ -n "$IS_WSL2" ];then KUBE_HOSTNAME='wsl2' ; fi

  for options in $@
  do
    test $options = '--skip-cp-k8s-bin' && export SKIP_CP_K8S_BIN=1
    [[ $options =~ --hostname=* ]] && KUBE_HOSTNAME=`echo $options | cut -d "=" -f 2`
  done

  echo $KUBE_HOSTNAME | grep -q '-' \
    &&  print_info "hostname $KUBE_HOSTNAME include -,
not allowed, please input new hostname by --hostname=XXX" \
    && exit 1 || true

  # if [ $HOSTNAME = "wsl2" ];then
  #   print_info "hostname not allowed set wsl2, please input new hostname by --hostname=XXX"
  #   exit 1
  # fi
}

_k8s_join_cert_checker(){
  ls systemd/certs/{ca.pem,\
kubectl.kubeconfig,\
kubelet.config.yaml,\
kube-proxy.config.yaml,\
csr-crb.yaml,\
kubectl.kubeconfig,\
kube-proxy.kubeconfig} > /dev/null
}

_k8s_join(){
  NODE_IP=$1

  # hostname
  _k8s_join_hostname_handler $@

  # cert
  _k8s_join_cert_checker

  # dep
  _k8s_install_dep

  $(_sudo) mkdir -p /etc/{docker,systemd/system}
  $(_sudo) chmod -R 755 /opt

  _k8s_install_disable_firewalld

  # cni
  _cni_install
  _cni_config

  _helm_install

  test $SKIP_CP_K8S_BIN -ne 1 && _k8s_install_cp_bin || \
  (print_info "skip cp k8s bin" ; \
     $(_sudo) install -m755 bin/*.sh ${K8S_ROOT}/bin/ ; \
     $(_sudo) chmod +x ${K8S_ROOT}/bin/*)

  _k8s_join_conf_cp

  _k8s_join_conf_replace
  _replace_kubelet_resolv_conf

  _gvisor_install
  _crictl_install
  _runc_install

  _cri_handler $2

  # _k8s_install_systemd kube-proxy kubelet kubelet@
  _k8s_install_systemd

  _k8s_install_mkdir

  _k8s_install_kubectl_conf
}

_local_up(){
  echo "
ETCD:

$ $(_sudo) systemctl start etcd

MASTER:

$ $(_sudo) systemctl start kube-apiserver
$ $(_sudo) systemctl start kube-controller-manager
$ $(_sudo) systemctl start kube-scheduler

NODE:

$ $(_sudo) systemctl start kube-proxy

# if cri is docker, please exec
$ $(_sudo) systemctl start kubelet

# if cri is containerd, please exec
$ $(_sudo) systemctl start kubelet@cri-containerd

# if cri is crio, please exec
$ $(_sudo) systemctl start kubelet@crio

$ $(_sudo) systemctl start docker (Docker runtime only)
$ $(_sudo) systemctl start cri-containerd@1.5 (Containerd runtime only)
$ $(_sudo) systemctl start cri-o (CRI-O runtime only)
"
}

_local_cleanup(){
  $(_sudo) systemctl stop etcd \
    kube-apiserver kube-controller-manager kube-scheduler \
    kube-proxy kubelet \
    docker cri-containerd@1.5 crio || true

  set +e
  $(_sudo) rm -rf ${K8S_ROOT}

  _docker_default_conf

  $(_sudo) ip addr flush cni0
  $(_sudo) ip addr flush docker0

  $(_sudo) rm -rf /etc/sysctl.d/99-kubernetes-cri.conf

  $(_sudo) rm -rf /etc/systemd/system/kubelet.service.d
  $(_sudo) rm -rf /etc/systemd/system/kubelet.service

  $(_sudo) systemctl daemon-reload || true
  $(_sudo) systemctl restart docker|| true
}

if [ "${LNMP_CN_ENV}" = 'false' ];then
  print_info "CN mirror disabled, I will replace cn mirror ... "
  $(_sudo) sed -i "s#hub-mirror.c.163.com#${LNMP_DOCKER_MIRROR:-mirror.gcr.io}#g" \
               `grep hub-mirror.c.163.com -rl $PWD/systemd` || true
  $(_sudo) sed -i "s#mirror.baidubce.com#${LNMP_DOCKER_MIRROR:-mirror.gcr.io}#g" \
               `grep mirror.baidubce.com -rl $PWD/systemd` || true
  $(_sudo) sed -i "s#registry.cn-hangzhou.aliyuncs.com/google_containers/pause#k8s.gcr.io/pause#g" \
               `grep registry.cn-hangzhou.aliyuncs.com/google_containers/pause -rl $PWD/systemd` || true
  # $(_sudo) sed -i "s#quay.mirror#quay.io#g" `grep quay.mirror -rl $PWD/systemd` || true
  $(_sudo) sed -i "s#ccr.ccs.tencentyun.com/khs1994/nginx-ingress-controller#k8s.gcr.io/ingress-nginx/controller#g" \
               `grep ccr.ccs.tencentyun.com/khs1994/nginx-ingress-controller -rl $PWD/addons` || true
  $(_sudo) sed -i "s#ccr.ccs.tencentyun.com/gcr-mirror/metrics-server#k8s.gcr.io/metrics-server/metrics-server#g" \
               `grep ccr.ccs.tencentyun.com/gcr-mirror/metrics-server -rl $PWD/addons` || true
  $(_sudo) sed -i "s#registry.cn-hangzhou.aliyuncs.com/google_containers/pause#k8s.gcr.io/pause#g" \
               `grep registry.cn-hangzhou.aliyuncs.com/google_containers/pause -rl $PWD/addons` || true
  print_info "[end] CN mirror disabled"
fi

_command=$1
shift

case $_command in
    kubernetes-server )
      _kubernetes_server_install $@
      ;;
    pull-k8s-image )
      _get_kubernetes_server_by_docker_image $@
      ;;
    kubectl-install )
      kubectl_install
      ;;
    kubectl-info )
      kubectl_info
      ;;
    local-install )
      _k8s_install $@
      ;;
    local-up )
      _local_up
      ;;
    local-cleanup )
      _local_cleanup
      ;;
    join )
      if [ -z $1 ];then echo "Please Input Node Ip"; exit 1; fi
      _k8s_join $@
      ;;
    nfs )
      # $(_sudo) modprobe {nfs,nfsd,rpcsec_gss_krb5} || true
      # $(_sudo) modprobe nfsd || true
      cd nfs-server
      if [ "$1" = 'down' ];then
        exec docker-compose stop nfs
      elif [ "$1" = 'logs' ]; then
        exec docker-compose logs nfs
      fi
      exec docker-compose up -d nfs
      ;;
    * )
      command -v $_command > /dev/null && $_command $@ || echo "command not found"
      ;;
esac
